Inspect Modelling Performance

Inspect Modelling Performance

suppressPackageStartupMessages(library(tidyverse))
library(targets)
library(DT)

knitr::opts_knit$set(root.dir = "../../")

VIF

df_cv_preds_and_coefs <- tar_read(df_cv_preds_and_coefs)

df_coefs <- df_cv_preds_and_coefs %>%
  select(year_start_act, category_id, pred) %>%
  mutate(coefs = map(pred, "coefs")) %>%
  select(- pred) %>%
  unnest(coefs) %>%
  unnest(coefs)

df_coefs %>%
  filter(year_start_act == 2019) %>%
  group_by(category_id) %>%
  summarise(max_vif = max(vif, na.rm = TRUE)) %>%
  knitr::kable()

category_id	max_vif
cnsn	1.253876
dtin	1.105846
ptpe	1.048755
sfty	1.966747
spno	1.045658

Perfermance Metrics

df_perf <- tar_read(df_perf)

AUC

df_perf %>%
  filter(.metric == "roc_auc") %>%
  knitr::kable(digits = 2)

category_id	.metric	mean	sd	n
cnsn	roc_auc	0.61	0.15	8
dtin	roc_auc	0.60	0.10	8
ptpe	roc_auc	0.59	0.06	8
sfty	roc_auc	0.63	0.07	8
spno	roc_auc	0.53	0.06	8

Brier

df_perf %>%
  filter(.metric == "brier") %>%
  knitr::kable(digits = 2)

category_id	.metric	mean	sd	n
cnsn	brier	0.24	0.03	8
dtin	brier	0.19	0.04	8
ptpe	brier	0.23	0.04	8
sfty	brier	0.25	0.03	8
spno	brier	0.24	0.03	8

Calibration

Linear Calibration

tar_read(df_calib) %>%
  select(- plot_data) %>%
  knitr::kable(digits = 3)

category_id	lower	base_rate	upper	intercept	slope
cnsn	0.250	0.455	0.593	0.171	0.603
dtin	0.426	0.726	0.849	0.374	0.506
ptpe	0.482	0.691	0.786	0.482	0.304
sfty	0.242	0.474	0.704	0.193	0.577
spno	0.628	0.637	0.637	0.654	-0.026

Plot

tar_read(p_calib)